import os
import sys
import shutil
import datetime
import requests
import numpy as np
import pandas as pd

from tools.setting import DATA_DIR
from tools.time_tool import date_shifter

sys.path.append('..')

API_KEY = '1zWq4Li0KlvJgokm8mJoXa8OeB2'  # dixinsuanlieth1@163.com

api_http = {
    'Addresses': {'Depositing Addresses': 'https://api.glassnode.com/v1/metrics/addresses/sending_to_exchanges_count',
                  'Withdrawing Addresses': 'https://api.glassnode.com/v1/metrics/addresses/receiving_from_exchanges_count',
                  'Total Addresses': 'https://api.glassnode.com/v1/metrics/addresses/count',
                  'Sending Addresses': 'https://api.glassnode.com/v1/metrics/addresses/sending_count',
                  'Receiving Addresses': 'https://api.glassnode.com/v1/metrics/addresses/receiving_count',
                  'Active Addresses': 'https://api.glassnode.com/v1/metrics/addresses/active_count',
                  'New Addresses': 'https://api.glassnode.com/v1/metrics/addresses/new_non_zero_count',
                  'Accumulation Addresses': 'https://api.glassnode.com/v1/metrics/addresses/accumulation_count',
                  'Accumulation Balance': 'https://api.glassnode.com/v1/metrics/addresses/accumulation_balance',
                  'Addresses with Non-Zero Balance': 'https://api.glassnode.com/v1/metrics/addresses/non_zero_count',
                  'Addresses with Balance ≥ 0.01': 'https://api.glassnode.com/v1/metrics/addresses/min_point_zero_1_count',
                  'Addresses with Balance ≥ 0.1': 'https://api.glassnode.com/v1/metrics/addresses/min_point_1_count',
                  'Addresses with Balance ≥ 1': 'https://api.glassnode.com/v1/metrics/addresses/min_1_count',
                  'Addresses with Balance ≥ 10': 'https://api.glassnode.com/v1/metrics/addresses/min_10_count',
                  'Addresses with Balance ≥ 100': 'https://api.glassnode.com/v1/metrics/addresses/min_100_count',
                  'Addresses with Balance ≥ 1k': 'https://api.glassnode.com/v1/metrics/addresses/min_1k_count',
                  'Addresses with Balance ≥ 10k': 'https://api.glassnode.com/v1/metrics/addresses/min_10k_count',
                  'Addresses with Balance ≥ 32 ETH': 'https://api.glassnode.com/v1/metrics/addresses/min_32_count',
                  'Supply Held by Addresses with Balance less 0.001': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_less_0001',
                  'Supply Held by Addresses with Balance 0.001 - 0.01': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_0001_001',
                  'Supply Held by Addresses with Balance 0.01 - 0.1': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_001_01',
                  'Supply Held by Addresses with Balance 0.1 - 1': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_01_1',
                  'Supply Held by Addresses with Balance 1 - 10': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_1_10',
                  'Supply Held by Addresses with Balance 10 - 100': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_10_100',
                  'Supply Held by Addresses with Balance 100 - 1k': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_100_1k',
                  'Supply Held by Addresses with Balance 1k - 10k': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_1k_10k',
                  'Supply Held by Addresses with Balance 10k - 100k': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_10k_100k',
                  'Supply Held by Addresses with Balance more 100k': 'https://api.glassnode.com/v1/metrics/addresses/supply_balance_more_100k',
                  'Address Supply Distribution': 'https://api.glassnode.com/v1/metrics/addresses/supply_distribution_relative',
                  'Addresses with Balance ≥ $1': 'https://api.glassnode.com/v1/metrics/addresses/min_1_usd_count',
                  'Addresses with Balance ≥ $10': 'https://api.glassnode.com/v1/metrics/addresses/min_10_usd_count',
                  'Addresses with Balance ≥ $100': 'https://api.glassnode.com/v1/metrics/addresses/min_100_usd_count',
                  'Addresses with Balance ≥ $1k': 'https://api.glassnode.com/v1/metrics/addresses/min_1k_usd_count',
                  'Addresses with Balance ≥ $10k': 'https://api.glassnode.com/v1/metrics/addresses/min_10k_usd_count',
                  'Addresses with Balance ≥ $100k': 'https://api.glassnode.com/v1/metrics/addresses/min_100k_usd_count',
                  'Addresses with Balance ≥ $1M': 'https://api.glassnode.com/v1/metrics/addresses/min_1m_usd_count',
                  'Addresses in Profit': 'https://api.glassnode.com/v1/metrics/addresses/profit_count',
                  'Addresses in Loss': 'https://api.glassnode.com/v1/metrics/addresses/loss_count',
                  'Percent Addresses in Profit': 'https://api.glassnode.com/v1/metrics/addresses/profit_relative'},
    'Indicators': {'RHODL Ratio': 'https://api.glassnode.com/v1/metrics/indicators/rhodl_ratio',
                   'CVDD': 'https://api.glassnode.com/v1/metrics/indicators/cvdd',
                   'Balanced Price': 'https://api.glassnode.com/v1/metrics/indicators/balanced_price_usd',
                   'Hash Ribbon': 'https://api.glassnode.com/v1/metrics/indicators/hash_ribbon',
                   'Difficulty Ribbon': 'https://api.glassnode.com/v1/metrics/indicators/difficulty_ribbon',
                   'Difficulty Ribbon Compression': 'https://api.glassnode.com/v1/metrics/indicators/difficulty_ribbon_compression',
                   'NVT Ratio': 'https://api.glassnode.com/v1/metrics/indicators/nvt',
                   'NVT Signal': 'https://api.glassnode.com/v1/metrics/indicators/nvts',
                   'Velocity': 'https://api.glassnode.com/v1/metrics/indicators/velocity',
                   'Entity-Adjusted NVT': 'https://api.glassnode.com/v1/metrics/indicators/nvt_entity_adjusted',
                   'Supply-Adjusted CDD': 'https://api.glassnode.com/v1/metrics/indicators/cdd_supply_adjusted',
                   'Binary CDD': 'https://api.glassnode.com/v1/metrics/indicators/cdd_supply_adjusted_binary',
                   'Supply-Adjusted Dormancy': 'https://api.glassnode.com/v1/metrics/indicators/average_dormancy_supply_adjusted',
                   'SOPD (ATH-Partitioned)': 'https://api.glassnode.com/v1/metrics/indicators/spent_output_price_distribution_ath',
                   'SOPD (Percent-Partitioned)': 'https://api.glassnode.com/v1/metrics/indicators/spent_output_price_distribution_percent',
                   'Puell Multiple': 'https://api.glassnode.com/v1/metrics/indicators/puell_multiple',
                   'aSOPR': 'https://api.glassnode.com/v1/metrics/indicators/sopr_adjusted',
                   'Reserve Risk': 'https://api.glassnode.com/v1/metrics/indicators/reserve_risk',
                   'STH-SOPR': 'https://api.glassnode.com/v1/metrics/indicators/sopr_less_155',
                   'LTH-SOPR': 'https://api.glassnode.com/v1/metrics/indicators/sopr_more_155',
                   'Hodler Net Position Change': 'https://api.glassnode.com/v1/metrics/indicators/hodler_net_position_change',
                   'Hodled or Lost Coins': 'https://api.glassnode.com/v1/metrics/indicators/hodled_lost_coins',
                   'Coin Years Destroyed (CYD)': 'https://api.glassnode.com/v1/metrics/indicators/cyd',
                   'Supply-Adjusted CYD': 'https://api.glassnode.com/v1/metrics/indicators/cyd_supply_adjusted',
                   'Entity-Adjusted CYD': 'https://api.glassnode.com/v1/metrics/indicators/cyd_account_based',
                   'Entity- and Supply-Adjusted CYD': 'https://api.glassnode.com/v1/metrics/indicators/cyd_account_based_supply_adjusted',
                   '90D Coin Days Destroyed (CDD-90)': 'https://api.glassnode.com/v1/metrics/indicators/cdd90_age_adjusted',
                   'Entity-Adjusted 90D Coin Days Destroyed (eCDD-90)': 'https://api.glassnode.com/v1/metrics/indicators/cdd90_account_based_age_adjusted',
                   'SOPR': 'https://api.glassnode.com/v1/metrics/indicators/sopr',
                   'Coin Days Destroyed (CDD)': 'https://api.glassnode.com/v1/metrics/indicators/cdd',
                   'ASOL': 'https://api.glassnode.com/v1/metrics/indicators/asol',
                   'MSOL': 'https://api.glassnode.com/v1/metrics/indicators/msol',
                   'Dormancy': 'https://api.glassnode.com/v1/metrics/indicators/average_dormancy',
                   'Liveliness': 'https://api.glassnode.com/v1/metrics/indicators/liveliness',
                   'Relative Unrealized Profit': 'https://api.glassnode.com/v1/metrics/indicators/unrealized_profit',
                   'Relative Unrealized Loss': 'https://api.glassnode.com/v1/metrics/indicators/unrealized_loss',
                   'Net Unrealized Profit Loss (NUPL)': 'https://api.glassnode.com/v1/metrics/indicators/net_unrealized_profit_loss',
                   'STH-NUPL': 'https://api.glassnode.com/v1/metrics/indicators/nupl_less_155',
                   'LTH-NUPL': 'https://api.glassnode.com/v1/metrics/indicators/nupl_more_155',
                   'Entity-Adjusted SOPR': 'https://api.glassnode.com/v1/metrics/indicators/sopr_account_based',
                   'Entity-Adjusted CDD': 'https://api.glassnode.com/v1/metrics/indicators/cdd_account_based',
                   'Entity-Adjusted ASOL': 'https://api.glassnode.com/v1/metrics/indicators/asol_account_based',
                   'Entity-Adjusted MSOL': 'https://api.glassnode.com/v1/metrics/indicators/msol_account_based',
                   'Entity-Adjusted Dormancy': 'https://api.glassnode.com/v1/metrics/indicators/dormancy_account_based',
                   'Entity-Adjusted Dormancy Flow': 'https://api.glassnode.com/v1/metrics/indicators/dormancy_flow',
                   'Entity-Adjusted Liveliness': 'https://api.glassnode.com/v1/metrics/indicators/liveliness_account_based',
                   'Entity-Adjusted MVRV': 'https://api.glassnode.com/v1/metrics/indicators/mvrv_account_based',
                   'Entity-Adjusted Realized Cap': 'https://api.glassnode.com/v1/metrics/indicators/rcap_account_based',
                   'Entity-Adjusted Unrealized Profit': 'https://api.glassnode.com/v1/metrics/indicators/unrealized_profit_account_based',
                   'Entity-Adjusted Unrealized Loss': 'https://api.glassnode.com/v1/metrics/indicators/unrealized_loss_account_based',
                   'Entity-Adjusted NUPL': 'https://api.glassnode.com/v1/metrics/indicators/net_unrealized_profit_loss_account_based',
                   'Entity-Adjusted STH-NUPL': 'https://api.glassnode.com/v1/metrics/indicators/nupl_less_155_account_based',
                   'Entity-Adjusted LTH-NUPL': 'https://api.glassnode.com/v1/metrics/indicators/nupl_more_155_account_based',
                   'Net Realized Profit Loss': 'https://api.glassnode.com/v1/metrics/indicators/net_realized_profit_loss',
                   'Realized P L Ratio': 'https://api.glassnode.com/v1/metrics/indicators/realized_profit_loss_ratio',
                   'Stock-to-Flow Ratio': 'https://api.glassnode.com/v1/metrics/indicators/stock_to_flow_ratio',
                   'Stock-to-Flow Deflection': 'https://api.glassnode.com/v1/metrics/indicators/stock_to_flow_deflection',
                   'Realized Profit': 'https://api.glassnode.com/v1/metrics/indicators/realized_profit',
                   'Realized Loss': 'https://api.glassnode.com/v1/metrics/indicators/realized_loss',
                   # SSR指标自己计算
                   'Stablecoin Supply Ratio (SSR)': 'https://api.glassnode.com/v1/metrics/indicators/ssr',
                   'Stablecoin Supply Ratio (SSR) Oscillator': 'https://api.glassnode.com/v1/metrics/indicators/ssr_oscillator',
                   'URPD (ATH-Partitioned)': 'https://api.glassnode.com/v1/metrics/indicators/utxo_realized_price_distribution_ath',
                   'URPD (Percent-Partitioned)': 'https://api.glassnode.com/v1/metrics/indicators/utxo_realized_price_distribution_percent',
                   'Spent Output Age Bands': 'https://api.glassnode.com/v1/metrics/indicators/soab',
                   'Spent Outputs less 1h': 'https://api.glassnode.com/v1/metrics/indicators/sol_1h',
                   'Spent Outputs 1h-24h': 'https://api.glassnode.com/v1/metrics/indicators/sol_1h_24h',
                   'Spent Outputs 1d-1w': 'https://api.glassnode.com/v1/metrics/indicators/sol_1d_1w',
                   'Spent Outputs 1w-1m': 'https://api.glassnode.com/v1/metrics/indicators/sol_1w_1m',
                   'Spent Outputs 1m-3m': 'https://api.glassnode.com/v1/metrics/indicators/sol_1m_3m',
                   'Spent Outputs 3m-6m': 'https://api.glassnode.com/v1/metrics/indicators/sol_3m_6m',
                   'Spent Outputs 6m-12m': 'https://api.glassnode.com/v1/metrics/indicators/sol_6m_12m',
                   'Spent Outputs 1y-2y': 'https://api.glassnode.com/v1/metrics/indicators/sol_1y_2y',
                   'Spent Outputs 2y-3y': 'https://api.glassnode.com/v1/metrics/indicators/sol_2y_3y',
                   'Spent Outputs 3y-5y': 'https://api.glassnode.com/v1/metrics/indicators/sol_3y_5y',
                   'Spent Outputs 5y-7y': 'https://api.glassnode.com/v1/metrics/indicators/sol_5y_7y',
                   'Spent Outputs 7y-10y': 'https://api.glassnode.com/v1/metrics/indicators/sol_7y_10y',
                   'Spent Outputs more 10y': 'https://api.glassnode.com/v1/metrics/indicators/sol_more_10y',
                   'Spent Volume Age Bands (SVAB)': 'https://api.glassnode.com/v1/metrics/indicators/svab',
                   'Spent Volume less 1h': 'https://api.glassnode.com/v1/metrics/indicators/svl_1h',
                   'Spent Volume 1h-24h': 'https://api.glassnode.com/v1/metrics/indicators/svl_1h_24h',
                   'Spent Volume 1d-1w': 'https://api.glassnode.com/v1/metrics/indicators/svl_1d_1w',
                   'Spent Volume 1w-1m': 'https://api.glassnode.com/v1/metrics/indicators/svl_1w_1m',
                   'Spent Volume 1m-3m': 'https://api.glassnode.com/v1/metrics/indicators/svl_1m_3m',
                   'Spent Volume 3m-6m': 'https://api.glassnode.com/v1/metrics/indicators/svl_3m_6m',
                   'Spent Volume 6m-12m': 'https://api.glassnode.com/v1/metrics/indicators/svl_6m_12m',
                   'Spent Volume 1y-2y': 'https://api.glassnode.com/v1/metrics/indicators/svl_1y_2y',
                   'Spent Volume 2y-3y': 'https://api.glassnode.com/v1/metrics/indicators/svl_2y_3y',
                   'Spent Volume 3y-5y': 'https://api.glassnode.com/v1/metrics/indicators/svl_3y_5y',
                   'Spent Volume 5y-7y': 'https://api.glassnode.com/v1/metrics/indicators/svl_5y_7y',
                   'Spent Volume 7y-10y': 'https://api.glassnode.com/v1/metrics/indicators/svl_7y_10y',
                   'Spent Volume more 10y': 'https://api.glassnode.com/v1/metrics/indicators/svl_more_10y',
                   'Bitcoin Volatility Index (BVIN)': 'https://api.glassnode.com/v1/metrics/indicators/bvin',
                   'Investor Capitalization': 'https://api.glassnode.com/v1/metrics/indicators/investor_capitalization',
                   'Realized Profits-to-Value (RPV) Ratio': 'https://api.glassnode.com/v1/metrics/indicators/realized_profits_to_value_ratio',
                   'Pi Cycle Top Indicator': 'https://api.glassnode.com/v1/metrics/indicators/pi_cycle_top'},
    'Market': {'Price Drawdown from ATH': 'https://api.glassnode.com/v1/metrics/market/price_drawdown_relative',
               'Delta Cap': 'https://api.glassnode.com/v1/metrics/market/deltacap_usd',
               'Market Cap': 'https://api.glassnode.com/v1/metrics/market/marketcap_usd',
               'MVRV Ratio': 'https://api.glassnode.com/v1/metrics/market/mvrv',
               'Realized Cap': 'https://api.glassnode.com/v1/metrics/market/marketcap_realized_usd',
               'MVRV Z-Score': 'https://api.glassnode.com/v1/metrics/market/mvrv_z_score',
               'STH-MVRV': 'https://api.glassnode.com/v1/metrics/market/mvrv_less_155',
               'LTH-MVRV': 'https://api.glassnode.com/v1/metrics/market/mvrv_more_155',
               'Realized Price': 'https://api.glassnode.com/v1/metrics/market/price_realized_usd'},
    'Mining': {'Difficulty': 'https://api.glassnode.com/v1/metrics/mining/difficulty_latest',
               'Hash Rate': 'https://api.glassnode.com/v1/metrics/mining/hash_rate_mean',
               'Miner Revenue (Total)': 'https://api.glassnode.com/v1/metrics/mining/revenue_sum',
               'Miner Revenue (Fees)': 'https://api.glassnode.com/v1/metrics/mining/revenue_from_fees',
               'Miner Revenue (Block Rewards)': 'https://api.glassnode.com/v1/metrics/mining/volume_mined_sum',
               'Miner Outflow Multiple': 'https://api.glassnode.com/v1/metrics/mining/miners_outflow_multiple',
               'Thermocap': 'https://api.glassnode.com/v1/metrics/mining/thermocap',
               'Market Cap to Thermocap Ratio': 'https://api.glassnode.com/v1/metrics/mining/marketcap_thermocap_ratio',
               'Miner Unspent Supply': 'https://api.glassnode.com/v1/metrics/mining/miners_unspent_supply'},
    'Derivatives': {
        # funding_rate 自己计算
        'Futures Perpetual Funding Rate': 'https://api.glassnode.com/v1/metrics/derivatives/futures_funding_rate_perpetual',
        'Futures Perpetual Funding Rate (All)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_funding_rate_perpetual_all',
        'Cash-Margined Futures Open Interest': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_cash_margin_sum',
        'Crypto-Margined Futures Open Interest': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_crypto_margin_sum',
        'Percent Crypto-Margined Futures Open Interest': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_crypto_margin_relative',
        'Futures Estimated Leverage Ratio': 'https://api.glassnode.com/v1/metrics/derivatives/futures_estimated_leverage_ratio',
        'Options ATM Implied Volatility (1 Week)': 'https://api.glassnode.com/v1/metrics/derivatives/options_atm_implied_volatility_1_week',
        'Options ATM Implied Volatility (1 Month)': 'https://api.glassnode.com/v1/metrics/derivatives/options_atm_implied_volatility_1_month',
        'Options ATM Implied Volatility (3 Months)': 'https://api.glassnode.com/v1/metrics/derivatives/options_atm_implied_volatility_3_months',
        'Options ATM Implied Volatility (6 Months)': 'https://api.glassnode.com/v1/metrics/derivatives/options_atm_implied_volatility_6_months',
        'Options ATM Implied Volatility (All)': 'https://api.glassnode.com/v1/metrics/derivatives/options_atm_implied_volatility_all',
        'Futures Volume': 'https://api.glassnode.com/v1/metrics/derivatives/futures_volume_daily_sum',
        'Futures Volume Perpetual': 'https://api.glassnode.com/v1/metrics/derivatives/futures_volume_daily_perpetual_sum',
        'Futures Open Interest': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_sum',
        'Futures Open Interest Perpetual': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_perpetual_sum',
        'Futures Short Liquidations (Total)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_liquidated_volume_short_sum',
        'Futures Short Liquidations (Mean)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_liquidated_volume_short_mean',
        'Futures Long Liquidations (Total)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_liquidated_volume_long_sum',
        'Futures Long Liquidations (Mean)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_liquidated_volume_long_mean',
        'Futures Long Liquidations Dominance': 'https://api.glassnode.com/v1/metrics/derivatives/futures_liquidated_volume_long_relative',
        'Futures Volume (Stacked)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_volume_daily_sum_all',
        'Futures Volume Perpetual (Stacked)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_volume_daily_perpetual_sum_all',
        'Futures Open Interest (Stacked)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_sum_all',
        'Futures Open Interest Perpetual (Stacked)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_perpetual_sum_all',
        'Options Volume': 'https://api.glassnode.com/v1/metrics/derivatives/options_volume_daily_sum',
        'Options Open Interest': 'https://api.glassnode.com/v1/metrics/derivatives/options_open_interest_sum',
        'Options OI by Strike Price': 'https://api.glassnode.com/v1/metrics/derivatives/options_open_interest_distribution',
        'Futures Term Structure': 'https://api.glassnode.com/v1/metrics/derivatives/futures_term_structure',
        'Futures Term Structure by Exchange': 'https://api.glassnode.com/v1/metrics/derivatives/futures_term_structure_by_exchange',
        'Futures Annualized Rolling Basis (3M)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_annualized_basis_3m',
        'Futures Open Interest (Current)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_open_interest_latest',
        'Futures Volume (Latest 24h)': 'https://api.glassnode.com/v1/metrics/derivatives/futures_volume_daily_latest'},
    'Distribution': {'Exchange Balance (Total)': 'https://api.glassnode.com/v1/metrics/distribution/balance_exchanges',
                     'Exchange Net Position Change': 'https://api.glassnode.com/v1/metrics/distribution/exchange_net_position_change',
                     'Exchange Balance (Percent)': 'https://api.glassnode.com/v1/metrics/distribution/balance_exchanges_relative',
                     'Exchange Balance (Stacked)': 'https://api.glassnode.com/v1/metrics/distribution/balance_exchanges_all',
                     'Miner Balance (Stacked)': 'https://api.glassnode.com/v1/metrics/distribution/balance_miners_all',
                     'Miner Net Position Change': 'https://api.glassnode.com/v1/metrics/distribution/balance_miners_change',
                     'OTC Desks Holdings': 'https://api.glassnode.com/v1/metrics/distribution/balance_otc_desks',
                     'Supply of Top 1% Addresses': 'https://api.glassnode.com/v1/metrics/distribution/balance_1pct_holders',
                     'Gini Coefficient': 'https://api.glassnode.com/v1/metrics/distribution/gini',
                     'Herfindahl Index': 'https://api.glassnode.com/v1/metrics/distribution/herfindahl',
                     'Supply in Smart Contracts': 'https://api.glassnode.com/v1/metrics/distribution/supply_contracts',
                     'Miner Balance': 'https://api.glassnode.com/v1/metrics/distribution/balance_miners_sum'},
    'Entities': {'Sending Entities': 'https://api.glassnode.com/v1/metrics/entities/sending_count',
                 'Receiving Entities': 'https://api.glassnode.com/v1/metrics/entities/receiving_count',
                 'Active Entities': 'https://api.glassnode.com/v1/metrics/entities/active_count',
                 'New Entities': 'https://api.glassnode.com/v1/metrics/entities/new_count',
                 'Entities Net Growth': 'https://api.glassnode.com/v1/metrics/entities/net_growth_count',
                 'Number of Whales': 'https://api.glassnode.com/v1/metrics/entities/min_1k_count',
                 'Supply Held by Entities with Balance less 0.001': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_less_0001',
                 'Supply Held by Entities with Balance 0.001 - 0.01': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_0001_001',
                 'Supply Held by Entities with Balance 0.01 - 0.1': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_001_01',
                 'Supply Held by Entities with Balance 0.1 - 1': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_01_1',
                 'Supply Held by Entities with Balance 1 - 10': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_1_10',
                 'Supply Held by Entities with Balance 10 - 100': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_10_100',
                 'Supply Held by Entities with Balance 100 - 1k': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_100_1k',
                 'Supply Held by Entities with Balance 1k - 10k': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_1k_10k',
                 'Supply Held by Entities with Balance 10k - 100k': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_10k_100k',
                 'Supply Held by Entities with Balance more 100k': 'https://api.glassnode.com/v1/metrics/entities/supply_balance_more_100k',
                 'Entities Supply Distribution': 'https://api.glassnode.com/v1/metrics/entities/supply_distribution_relative',
                 'Percent Entities in Profit': 'https://api.glassnode.com/v1/metrics/entities/profit_relative'},
    'Supply': {'Liquid and Illiquid Supply': 'https://api.glassnode.com/v1/metrics/supply/liquid_illiquid_sum',
               'Liquid Supply': 'https://api.glassnode.com/v1/metrics/supply/liquid_sum',
               'Highly Liquid Supply': 'https://api.glassnode.com/v1/metrics/supply/highly_liquid_sum',
               'Illiquid Supply': 'https://api.glassnode.com/v1/metrics/supply/illiquid_sum',
               'Liquid Supply Change': 'https://api.glassnode.com/v1/metrics/supply/liquid_change',
               'Illiquid Supply Change': 'https://api.glassnode.com/v1/metrics/supply/illiquid_change',
               'Circulating Supply': 'https://api.glassnode.com/v1/metrics/supply/current',
               'Minted Supply': 'https://api.glassnode.com/v1/metrics/supply/minted',
               'Burned Supply': 'https://api.glassnode.com/v1/metrics/supply/burned',
               'Issuance': 'https://api.glassnode.com/v1/metrics/supply/issued',
               'Inflation Rate': 'https://api.glassnode.com/v1/metrics/supply/inflation_rate',
               'Revived Supply 1+ Years': 'https://api.glassnode.com/v1/metrics/supply/revived_more_1y_sum',
               'Revived Supply 2+ Years': 'https://api.glassnode.com/v1/metrics/supply/revived_more_2y_sum',
               'Revived Supply 3+ Years': 'https://api.glassnode.com/v1/metrics/supply/revived_more_3y_sum',
               'Revived Supply 5+ Years': 'https://api.glassnode.com/v1/metrics/supply/revived_more_5y_sum',
               'Supply Last Active less 24h': 'https://api.glassnode.com/v1/metrics/supply/active_24h',
               'Supply Last Active 1d-1w': 'https://api.glassnode.com/v1/metrics/supply/active_1d_1w',
               'Supply Last Active 1w-1m': 'https://api.glassnode.com/v1/metrics/supply/active_1w_1m',
               'Supply Last Active 1m-3m': 'https://api.glassnode.com/v1/metrics/supply/active_1m_3m',
               'Supply Last Active 3m-6m': 'https://api.glassnode.com/v1/metrics/supply/active_3m_6m',
               'Supply Last Active 6m-12m': 'https://api.glassnode.com/v1/metrics/supply/active_6m_12m',
               'Supply Last Active 1y-2y': 'https://api.glassnode.com/v1/metrics/supply/active_1y_2y',
               'Supply Last Active 2y-3y': 'https://api.glassnode.com/v1/metrics/supply/active_2y_3y',
               'Supply Last Active 3y-5y': 'https://api.glassnode.com/v1/metrics/supply/active_3y_5y',
               'Supply Last Active 5y-7y': 'https://api.glassnode.com/v1/metrics/supply/active_5y_7y',
               'Supply Last Active 7y-10y': 'https://api.glassnode.com/v1/metrics/supply/active_7y_10y',
               'Supply Last Active more 10y': 'https://api.glassnode.com/v1/metrics/supply/active_more_10y',
               'HODL Waves': 'https://api.glassnode.com/v1/metrics/supply/hodl_waves',
               'Supply Last Active 1+ Years Ago': 'https://api.glassnode.com/v1/metrics/supply/active_more_1y_percent',
               'Supply Last Active 2+ Years Ago': 'https://api.glassnode.com/v1/metrics/supply/active_more_2y_percent',
               'Supply Last Active 3+ Years Ago': 'https://api.glassnode.com/v1/metrics/supply/active_more_3y_percent',
               'Supply Last Active 5+ Years Ago': 'https://api.glassnode.com/v1/metrics/supply/active_more_5y_percent',
               'Realized Cap HODL Waves': 'https://api.glassnode.com/v1/metrics/supply/rcap_hodl_waves',
               'Short to Long-Term Realized Value (SLRV) Ratio': 'https://api.glassnode.com/v1/metrics/supply/sth_lth_realized_value_ratio',
               'Adjusted Supply': 'https://api.glassnode.com/v1/metrics/supply/current_adjusted',
               'Supply in Profit': 'https://api.glassnode.com/v1/metrics/supply/profit_sum',
               'Supply in Loss': 'https://api.glassnode.com/v1/metrics/supply/loss_sum',
               'Percent Supply in Profit': 'https://api.glassnode.com/v1/metrics/supply/profit_relative',
               'Short-Term Holder Supply': 'https://api.glassnode.com/v1/metrics/supply/sth_sum',
               'Long-Term Holder Supply': 'https://api.glassnode.com/v1/metrics/supply/lth_sum',
               'Short-Term Holder Supply in Loss': 'https://api.glassnode.com/v1/metrics/supply/sth_loss_sum',
               'Long-Term Holder Supply in Loss': 'https://api.glassnode.com/v1/metrics/supply/lth_loss_sum',
               'Short-Term Holder Supply in Profit': 'https://api.glassnode.com/v1/metrics/supply/sth_profit_sum',
               'Long-Term Holder Supply in Profit': 'https://api.glassnode.com/v1/metrics/supply/lth_profit_sum',
               'Relative Long Short-Term Holder Supply': 'https://api.glassnode.com/v1/metrics/supply/lth_sth_profit_loss_relative',
               'Long-Term Holder Position Change': 'https://api.glassnode.com/v1/metrics/supply/lth_net_change',
               'Short-Term Holder Profit/Loss Ratio': 'https://api.glassnode.com/v1/metrics/supply/sth_profit_loss_ratio'},
    'Transactions': {'Transaction Count': 'https://api.glassnode.com/v1/metrics/transactions/count',
                     'Transaction Rate': 'https://api.glassnode.com/v1/metrics/transactions/rate',
                     'Contract Calls (External)': 'https://api.glassnode.com/v1/metrics/transactions/contract_calls_external_count',
                     'Contract Calls (Internal)': 'https://api.glassnode.com/v1/metrics/transactions/contract_calls_internal_count',
                     'Transaction Size (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/size_mean',
                     'Transaction Size (Total)': 'https://api.glassnode.com/v1/metrics/transactions/size_sum',
                     'Total Transfer Volume by Size (Entity-Adjusted)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_by_size_entity_adjusted_sum',
                     'Relative Transfer Volume by Size (Entity-Adjusted)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_by_size_entity_adjusted_relative',
                     'OTC Desks Incoming Transactions': 'https://api.glassnode.com/v1/metrics/transactions/transfers_to_otc_desks_count',
                     'OTC Desks Outgoing Transactions': 'https://api.glassnode.com/v1/metrics/transactions/transfers_from_otc_desks_count',
                     'OTC Desks Inflows': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_to_otc_desks_sum',
                     'OTC Desks Outflows': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_from_otc_desks_sum',
                     'Transfer Count': 'https://api.glassnode.com/v1/metrics/transactions/transfers_count',
                     'Transfer Rate': 'https://api.glassnode.com/v1/metrics/transactions/transfers_rate',
                     'Transfer Volume (Total)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_sum',
                     'Transfer Volume (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_mean',
                     'Transfer Volume (Median)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_median',
                     'Change-Adjusted Volume (Total)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_adjusted_sum',
                     'Change-Adjusted Volume (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_adjusted_mean',
                     'Change-Adjusted Volume (Median)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_adjusted_median',
                     'Percent Volume in Profit': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_profit_relative',
                     'Transfer Volume in Profit': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_profit_sum',
                     'Transfer Volume in Loss': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_loss_sum',
                     'Entity-Adjusted Transaction Count': 'https://api.glassnode.com/v1/metrics/transactions/entity_adjusted_count',
                     'Entity-Adjusted Volume (Total)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_entity_adjusted_sum',
                     'Entity-Adjusted Volume (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_entity_adjusted_mean',
                     'Entity-Adjusted Volume (Median)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_entity_adjusted_median',
                     'Miner Outgoing Transfers': 'https://api.glassnode.com/v1/metrics/transactions/transfers_from_miners_count',
                     'Miner Incoming Transfers': 'https://api.glassnode.com/v1/metrics/transactions/transfers_to_miners_count',
                     'Miner Outflow Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_from_miners_sum',
                     'Miner Inflow Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_to_miners_sum',
                     'Miner Netflow Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_miners_net',
                     'Inter-Exchange Transfers': 'https://api.glassnode.com/v1/metrics/transactions/transfers_between_exchanges_count',
                     'Inter-Exchange Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_between_exchanges_sum',
                     'Miners to Exchanges': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_miners_to_exchanges',
                     'Miners to Exchanges (Stacked)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_miners_to_exchanges_all',
                     'Exchange Inflow Volume (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_to_exchanges_mean',
                     'Exchange Outflow Volume (Mean)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_from_exchanges_mean',
                     'Exchange Net Flow Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_exchanges_net',
                     'Exchange Deposits': 'https://api.glassnode.com/v1/metrics/transactions/transfers_to_exchanges_count',
                     'Exchange Withdrawals': 'https://api.glassnode.com/v1/metrics/transactions/transfers_from_exchanges_count',
                     'Exchange Inflow Volume (Total)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_to_exchanges_sum',
                     'Exchange Outflow Volume (Total)': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_from_exchanges_sum',
                     'In-house Exchange Volume': 'https://api.glassnode.com/v1/metrics/transactions/transfers_volume_within_exchanges_sum',
                     },
    'Mempool': {
        'Mempool Transaction Count by Cohort': 'https://api.glassnode.com/v1/metrics/mempool/txs_count_distribution',
        'Mempool Transaction Count': 'https://api.glassnode.com/v1/metrics/mempool/txs_count_sum',
        'Mempool Transactions Size by Cohort (vByte)': 'https://api.glassnode.com/v1/metrics/mempool/txs_size_distribution',
        'Mempool Transactions Size (vByte)': 'https://api.glassnode.com/v1/metrics/mempool/txs_size_sum',
        'Mempool Total Amount of Fees by Cohort (vByte)': 'https://api.glassnode.com/v1/metrics/mempool/fees_distribution',
        'Mempool Total Amount of Fees': 'https://api.glassnode.com/v1/metrics/mempool/fees_sum',
        'Mempool Total Amount of Coins by Cohort (vByte)': 'https://api.glassnode.com/v1/metrics/mempool/txs_value_distribution',
        'Mempool Total Amount of Coins': 'https://api.glassnode.com/v1/metrics/mempool/txs_value_sum',
        'Mempool Average Relative Fee': 'https://api.glassnode.com/v1/metrics/mempool/fees_average_relative',
        'Mempool Median Relative Fee': 'https://api.glassnode.com/v1/metrics/mempool/fees_median_relative'},
    'ETH 2.0': {'New Deposits': 'https://api.glassnode.com/v1/metrics/eth2/staking_deposits_count',
                'New Value Staked': 'https://api.glassnode.com/v1/metrics/eth2/staking_volume_sum',
                'New Validators': 'https://api.glassnode.com/v1/metrics/eth2/staking_validators_count',
                'Total Number of Deposits': 'https://api.glassnode.com/v1/metrics/eth2/staking_total_deposits_count',
                'Total Value Staked': 'https://api.glassnode.com/v1/metrics/eth2/staking_total_volume_sum',
                'Total Number of Validators': 'https://api.glassnode.com/v1/metrics/eth2/staking_total_validators_count',
                'Phase 0 Staking Goal': 'https://api.glassnode.com/v1/metrics/eth2/staking_phase_0_goal_percent'},
    'DeFi': {'Total Value Locked in DeFi (TVL)': 'https://api.glassnode.com/v1/metrics/defi/total_value_locked'},
    'Fees': {'Gas Used (Total)': 'https://api.glassnode.com/v1/metrics/fees/gas_used_sum',
             'Gas Used (Mean)': 'https://api.glassnode.com/v1/metrics/fees/gas_used_mean',
             'Gas Used (Median)': 'https://api.glassnode.com/v1/metrics/fees/gas_used_median',
             'Transaction Gas Limit (Mean)': 'https://api.glassnode.com/v1/metrics/fees/gas_limit_tx_mean',
             'Transaction Gas Limit (Median)': 'https://api.glassnode.com/v1/metrics/fees/gas_limit_tx_median',
             'Gas Price (Mean)': 'https://api.glassnode.com/v1/metrics/fees/gas_price_mean',
             'Gas Price (Median)': 'https://api.glassnode.com/v1/metrics/fees/gas_price_median',
             'Fees (Total)': 'https://api.glassnode.com/v1/metrics/fees/volume_sum',
             'Fees (Mean)': 'https://api.glassnode.com/v1/metrics/fees/volume_mean',
             'Fees (Median)': 'https://api.glassnode.com/v1/metrics/fees/volume_median',
             'Fee Ratio Multiple (FRM)': 'https://api.glassnode.com/v1/metrics/fees/fee_ratio_multiple',
             'Exchange Fee Dominance': 'https://api.glassnode.com/v1/metrics/fees/exchanges_relative',
             'Exchange Fees (Total)': 'https://api.glassnode.com/v1/metrics/fees/exchanges_sum',
             'Exchange Fees (Mean)': 'https://api.glassnode.com/v1/metrics/fees/exchanges_mean'},
    'Institutions': {'Grayscale Holdings': 'https://api.glassnode.com/v1/metrics/institutions/grayscale_holdings_sum',
                     'Grayscale Flows': 'https://api.glassnode.com/v1/metrics/institutions/grayscale_flows_sum',
                     'Grayscale Premium': 'https://api.glassnode.com/v1/metrics/institutions/grayscale_premium_percent',
                     'Grayscale AUM': 'https://api.glassnode.com/v1/metrics/institutions/grayscale_aum_sum',
                     'Grayscale Market Price': 'https://api.glassnode.com/v1/metrics/institutions/grayscale_market_price_usd',
                     'Purpose Bitcoin ETF Holdings': 'https://api.glassnode.com/v1/metrics/institutions/purpose_etf_holdings_sum',
                     'Purpose Bitcoin ETF Flows': 'https://api.glassnode.com/v1/metrics/institutions/purpose_etf_flows_sum',
                     'The Bitcoin Fund (QBTC) Holdings': 'https://api.glassnode.com/v1/metrics/institutions/qbtc_holdings_sum',
                     'The Bitcoin Fund (QBTC) Flows': 'https://api.glassnode.com/v1/metrics/institutions/qbtc_flows_sum',
                     'The Bitcoin Fund (QBTC) Premium': 'https://api.glassnode.com/v1/metrics/institutions/qbtc_premium_percent',
                     'The Bitcoin Fund (QBTC) AUM': 'https://api.glassnode.com/v1/metrics/institutions/qbtc_aum_sum',
                     'The Bitcoin Fund (QBTC) Market Price': 'https://api.glassnode.com/v1/metrics/institutions/qbtc_market_price_usd',
                     'The Bitcoin Fund (QBTC.U) Holdings': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_holdings_sum',
                     'The Bitcoin Fund (QBTC.U) Flows': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_flows_sum',
                     'The Bitcoin Fund (QBTC.U) Premium': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_premium_percent',
                     'The Bitcoin Fund (QBTC.U) Outstanding Units': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_outstanding_units_sum',
                     'The Bitcoin Fund (QBTC.U) AUM': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_aum_sum',
                     'The Bitcoin Fund (QBTC.U) Market Price': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qbtc_market_price_usd',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) Holdings': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_holdings_sum',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) Flows': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_flows_sum',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) Premium': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_premium_percent',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) Outstanding Units': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_outstanding_units_sum',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) AUM': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_aum_sum',
                     '3iQ CoinShares Bitcoin ETF (BTCQ.U) Market Price': 'https://api.glassnode.com/v1/metrics/institutions/3iq_btcq_market_price_usd',
                     'The Ether Fund (QETH.U) Holdings': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_holdings_sum',
                     'The Ether Fund (QETH.U) Flows': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_flows_sum',
                     'The Ether Fund (QETH.U) Premium': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_premium_percent',
                     'The Ether Fund (QETH.U) Outstanding Units': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_outstanding_units_sum',
                     'The Ether Fund (QETH.U) AUM': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_aum_sum',
                     'The Ether Fund (QETH.U) Market Price': 'https://api.glassnode.com/v1/metrics/institutions/3iq_qeth_market_price_usd',
                     '3iQ CoinShares Ether ETF (ETHQ.U) Holdings': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_holdings_sum',
                     '3iQ CoinShares Ether ETF (ETHQ.U) Flows': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_flows_sum',
                     '3iQ CoinShares Ether ETF (ETHQ.U) Premium': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_premium_percent',
                     '3iQ CoinShares Ether ETF (ETHQ.U) Outstanding Units': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_outstanding_units_sum',
                     '3iQ CoinShares Ether ETF (ETHQ.U) AUM': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_aum_sum',
                     '3iQ CoinShares Ether ETF (ETHQ.U) Market Price': 'https://api.glassnode.com/v1/metrics/institutions/3iq_ethq_market_price_usd'},
    'Lightning': {
        'Lightning Network Channel Size (Mean)': 'https://api.glassnode.com/v1/metrics/lightning/channel_size_mean',
        'Lightning Network Channel Size (Median)': 'https://api.glassnode.com/v1/metrics/lightning/channel_size_median',
        'Lightning Network Capacity': 'https://api.glassnode.com/v1/metrics/lightning/network_capacity_sum',
        'Lightning Network Number of Channels': 'https://api.glassnode.com/v1/metrics/lightning/channels_count',
        'Lightning Network Number of Nodes': 'https://api.glassnode.com/v1/metrics/lightning/nodes_count'},
    'Protocols': {'Uniswap Transactions': 'https://api.glassnode.com/v1/metrics/protocols/uniswap_transaction_count',
                  'Uniswap Liquidity': 'https://api.glassnode.com/v1/metrics/protocols/uniswap_liquidity_latest',
                  'Uniswap Volume': 'https://api.glassnode.com/v1/metrics/protocols/uniswap_volume_sum'},
    'Blockchain': {'UTXOs (Total)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_count',
                   'UTXOs Created': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_created_count',
                   'UTXOs Spent': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_spent_count',
                   'UTXO Value Created (Total)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_created_value_sum',
                   'UTXO Value Spent (Total)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_spent_value_sum',
                   'UTXO Value Created (Mean)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_created_value_mean',
                   'UTXO Value Spent (Mean)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_spent_value_mean',
                   'UTXO Value Created (Median)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_created_value_median',
                   'UTXO Value Spent (Median)': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_spent_value_median',
                   'UTXOs in Profit': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_profit_count',
                   'UTXOs in Loss': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_loss_count',
                   'Percent UTXOs in Profit': 'https://api.glassnode.com/v1/metrics/blockchain/utxo_profit_relative',
                   'Block Height': 'https://api.glassnode.com/v1/metrics/blockchain/block_height',
                   'Blocks Mined': 'https://api.glassnode.com/v1/metrics/blockchain/block_count',
                   'Block Interval (Mean)': 'https://api.glassnode.com/v1/metrics/blockchain/block_interval_mean',
                   'Block Interval (Median)': 'https://api.glassnode.com/v1/metrics/blockchain/block_interval_median',
                   'Block Size (Mean)': 'https://api.glassnode.com/v1/metrics/blockchain/block_size_mean',
                   'Block Size (Total)': 'https://api.glassnode.com/v1/metrics/blockchain/block_size_sum'}
}

all_http = {}
for i, val in enumerate(list(api_http.values())):
    all_http.update(val)
indicator_list = list(all_http.keys())


def xlsx2csv_folder(source, dist):
    for file in os.listdir(source):
        if file.endswith('.xlsx'):
            if os.path.exists(os.path.join(dist, file.replace('.xlsx', '.csv'))):
                continue
            try:
                df = pd.read_excel(os.path.join(source, file))
                df.to_csv(os.path.join(dist, file.replace('.xlsx', '.csv')), index=False)
            except Exception as e:
                print(file)

def csv2xlsx_for_report(file_list, dist, log_price):
    report_files = []
    for file in file_list:
        print(file)
        df = pd.read_csv(file)
        dist_file = os.path.join(dist, file.replace('.csv', '.xlsx').split('\\')[-1])
        report_files.append(dist_file)
        if 'end_date' in df.columns:
            df.drop_duplicates(subset='end_date',inplace=True, keep='last')
            df.set_index('end_date', inplace=True)
        elif 'date' in df.columns:
            df.drop_duplicates(subset='date',inplace=True, keep='last')
            df.set_index('date', inplace=True)
        elif 'atime' in df.columns:
            df.drop_duplicates(subset='atime',inplace=True, keep='last')
            df.set_index('atime', inplace=True)
        else:
            df.to_excel(dist_file, index=False)
            continue
        if 'log_price' in df.columns:
            del df['log_price']
        if 'log_prices' in df.columns:
            del df['log_prices']
        df = pd.concat([df, log_price], join='inner', axis=1)
        df = df[df.index >= '2017-01-01']
        df.sort_index(inplace=True)
        df.to_excel(dist_file)
    return report_files

def report_file_aggregation(file_list, dist):
    report_files = []
    for file in file_list:
        dist_file = os.path.join(dist, file.split('\\')[-1])
        report_files.append(dist_file)
        shutil.copy(file, dist_file)
    return report_files

def filter_coins_not_updated_over_1year():
    ohlcvm_folder = 'E:\\data\\ohlcvm_data\\daily'
    today = datetime.datetime.now()
    flag_date = (today - datetime.timedelta(days=380)).strftime('%Y-%m-%d')
    coins_list = []
    for file in os.listdir(ohlcvm_folder):
        df = pd.read_csv(os.path.join(ohlcvm_folder, file))
        if len(df) == 0:
            continue
        if df['end_date'].iloc[-1] < flag_date:
            coins_list.append(file.split('.')[0])
    return coins_list


def check_https(all_http):
    start_date = int(datetime.datetime.timestamp(datetime.datetime.today() - datetime.timedelta(days=5)))
    parameters = {'a': 'BTC', 'api_key': API_KEY, 's': start_date}
    available_http = {}
    for i in all_http:
        link = all_http[i]
        try:
            res = requests.get(link, params=parameters)
            if res.status_code == 404:
                print(f'指标{i}数据获取失败，{res.status_code}; {res.text}')
            else:
                available_http[i] = link
        except Exception as e:
            print(f'指标{i}数据获取异常， {str(e)}')
    return available_http


def reformat_timestamp(df):
    data = df.copy()
    timezone = '+0000'
    data['datetime'] = pd.to_datetime(data['timestamp'], unit='ms').dt.tz_localize('UTC').dt.tz_convert(timezone)
    data['datetime'] = data['datetime'].dt.strftime('%Y-%m-%d %H:%M:%S')
    data.set_index('datetime', inplace=True)
    return data


def drawdown_from_ath(df):
    exp_max = df.close.expanding().max()
    mdd = (exp_max - df.close) / exp_max
    mdd.name = 'MaxDrawdown'
    return mdd


def segmented_maxdrawdown(data, segment_labels=None):
    data_list = []
    mdd_list = []
    if segment_labels is None:
        data_list.append(data)
    else:
        # data_list.append(data.loc[:segment_labels[0]])
        data_list.append(data[data.index < segment_labels[0]])
        for i in range(len(segment_labels) - 1):
            # data_list.append(data.loc[segment_labels[i] : segment_labels[i + 1]])
            data_list.append(data[data.index >= segment_labels[i] & (data.index < segment_labels[i + 1])])
        data_list.append(data.loc[segment_labels[-1]:])
    for _data in data_list:
        mdd_list.append(drawdown_from_ath(_data))
    mdd = pd.concat(mdd_list)
    return mdd


def max_drawdown_all(data):
    exp_max = data.expanding().max()
    mdd = (exp_max - data) / exp_max
    return mdd


def segmented_maxdrawdown_all(data, segment_labels=None):
    data_list = []
    mdd_list = []
    if segment_labels is None:
        data_list.append(data)
    else:
        # data_list.append(data.loc[:segment_labels[0]])
        data_list.append(data[data.index < segment_labels[0]])
        for i in range(len(segment_labels) - 1):
            # data_list.append(data.loc[segment_labels[i] : segment_labels[i + 1]])
            data_list.append(data[data.index >= segment_labels[i] & (data.index < segment_labels[i + 1])])
        data_list.append(data.loc[segment_labels[-1]:])
    for _data in data_list:
        mdd_list.append(max_drawdown_all(_data))
    mdd = pd.concat(mdd_list)
    return mdd


def get_all_data():
    folder = 'E:\\data\\ohlcvm_data\\daily'
    files = os.listdir(folder)
    df_dict = {i[:-4]:pd.read_csv(os.path.join(folder, i), index_col='end_date') for i in files}
    price_list = []
    amount_list = []
    marketcap_list = []
    for k, v in df_dict.items():
        _price = v['close']
        _price.name = k
        price_list.append(_price)
        _amount = v['amount']
        _amount.name = k
        amount_list.append(_amount)
        _marketcap = v['market_cap']
        _marketcap.name = k
        marketcap_list.append(_marketcap)
    df_price = pd.concat(price_list, axis=1)
    df_price.sort_index(inplace=True)
    df_amount = pd.concat(amount_list, axis=1)
    df_amount.sort_index(inplace=True)
    df_marketcap = pd.concat(marketcap_list, axis=1)
    df_marketcap.sort_index(inplace=True)
    return df_price, df_amount, df_marketcap


def get_analysis_coin_list(all_prices_df):
    # 剔除各种不需要分析的币，得到最终需要分析的币的名单

    # 剔除稳定币
    all_coins_name = list(all_prices_df.columns)
    file_path = os.path.join(DATA_DIR, 'coins_information')
    file_name = os.path.join(file_path, f'stablecoins_information.csv')
    all_stablecoin_list_df = pd.read_csv(file_name, index_col='slug')
    stablecoin_names = list(all_stablecoin_list_df.index)
    all_coins_name_without_stablecoins = [i for i in all_coins_name if i not in stablecoin_names]

    # 有些币是wrapped，其实走势是一致的，需要剔除掉
    # 剔除掉带wrapped-前缀的
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if 'wrapped-' not in i]
    # 剔除掉带staked的
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if 'staked' not in i]
    # 发布在不同链上的同一代币
    differentchain_similar_coin_list = ['bitcoin-bep2', 'sheesha-finance-bep20',
                                        'sheesha-finance-polygon', 'astronaut-polygon', 'atlantis-loans-polygon', 'mm-finance-polygon', 'mm-optimizer-polygon',
                                        'phenix-finance-polygon', 'sheesha-finance-polygon', 'waultswap-polygon', 'wolf-safe-poor-people-polygon']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in differentchain_similar_coin_list]

    # 换了名字的币
    differentname_coin_list = ['certick']  # certik改名为shentu
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in differentname_coin_list]

    # 剔除掉一些特殊的wrapped的币
    similar_coin_list = ['algomint-gobtc', 'bitcoin-additional', 'bitcoin-avalanche-bridged', 'btc-proxy', 'huobi-btc', 'interbtc', 'kintsugi-btc', 'oec-btc', 'renbtc', 'rsk-smart-bitcoin', 'sbtc', 'tbtc',
                         'wbnb', 'stader-bnbx', 'stkbnb',
                         'steth', 'aave-ethereum', 'beacon-eth', 'ethk', 'goeth', 'lido-finance-wsteth', 'peth', 'rocket-pool-eth', 'seth', 'seth2', 'stakewise-reth2', 'waneth', 'weth', 'ankreth',
                         'lido-staked-matic', 'stader-maticx', 'wmatic', 'xmatic', 'ankr-avalanche-reward-earning-bond', 'ankr-reward-earning-matic',
                         'usdj', 'aave-busd', 'aave-dai', 'axlusdc', 'balancer-boosted-aave-usdc', 'curve-fi-frax-usdc', 'curve-fi-gdai-gusdc-gusdt', 'orbit-bridge-klaytn-usdc', 'usd-coin-avalanche-bridged-usdc-e', 'wxdai',
                         'inverse-bitcoin-volatility-token', 'crypto-volatility-token']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in similar_coin_list]
    # 剔除掉一些杠杆代币
    leverage_coin_list = ['btcdown', 'dotdown', 'eth-down', 'ethdown', 'linkdown', 'trxdown', 'xrpdown', 'adadown', 'bnbdown',
                          'btcup', 'dotup', 'eth-up', 'ethup', 'linkup', 'trxup', 'xrpup', 'adaup', 'bnbup']
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if i not in leverage_coin_list]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-short-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-long-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-leverage-' not in i]
    # 剔除一些指数代币
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-index' not in i]
    # 剔除一些股票代币
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-stock-' not in i]
    all_coins_name_without_stablecoins = [i for i in all_coins_name_without_stablecoins if '-tokenized-' not in i]

    return all_coins_name_without_stablecoins


def prices_cleaning(prices):
    prices.replace(0, np.nan, inplace=True)
    try:
        temp_prices = prices[prices[~prices.isnull()].index[0]:prices[~prices.isnull()].index[-1]]  # 前后为NAN的都剔除，中间的不剔除
    except Exception as err:
        if 'index 0 is out of bounds' in str(err):
            # print(f'{prices.name} prices序列全为Nan, 请检查价格序列')
            pass
        return prices
    temp_prices = temp_prices.ffill()
    prices[temp_prices.index] = temp_prices
    return prices


def ico_date(prices):
    try:
        ico_date = prices[~prices.isnull()].index[0]
    except Exception as err:
        ico_date = np.nan
    return pd.Series([ico_date])


def ret_cleaning_process(split_df, **kwargs):
    all_amount_df = kwargs.pop('all_amount_df')
    total_amount_series = kwargs.pop('total_amount_series')
    ico_date_series = kwargs.pop('ico_date_series')

    # 把所有需要进行小额交易过滤的收益率转换成三列，一列为日期，一列为收益率，一列为币名称
    all_ret_series_list = []
    for i in split_df.columns:
        temp_ret_series = split_df[i]
        temp_ret_series.dropna(inplace=True)
        temp_ret_series.name = 'ret'
        temp_temp_ret_df = pd.DataFrame(temp_ret_series)
        temp_temp_ret_df['name'] = i
        all_ret_series_list.append(temp_temp_ret_df)
    temp_all_ret_df = pd.concat(all_ret_series_list, axis=0) # 不做排序的话，则按照时间顺序处理
    temp_all_ret_df.reset_index(inplace=True)

    # '2017-09-01'至 '2017-10-01'期间因为交易所清退，交易额可能存在异常情况，直接忽略这一段收益率，不进行过滤
    kick_out_date = pd.date_range('2017-09-01', '2017-10-01', freq='D').tolist()
    kick_out_date = [x.strftime("%Y-%m-%d") for x in kick_out_date]

    # 收益率进行小额交易过滤
    all_ret_df = split_df.copy()
    for index, this_row in temp_all_ret_df.iterrows():  # 逐个异常收益率进行分析
        this_date = this_row['end_date']  # 该收益率对应的日期
        previous_60_day = date_shifter(this_date, step='days', how_many=-60)  # 前60天对应日期
        this_name = this_row['name']  # 这个币的名称
        ico_date = ico_date_series[this_name]  # 这个币的ICO日期
        ico_next_7_days = date_shifter(ico_date, step='days', how_many=7)  # ICO的后7天

        if previous_60_day < ico_next_7_days:  # 如果前60天超过了该币的的ICO7天后，则小额交易过滤起始日期为ICO7天后
            previous_60_day = ico_next_7_days  # ico后7天交易额不做限制
        this_coin_recent_amount = all_amount_df.loc[previous_60_day:this_date, this_name]  # todo 该收益率前60天收益率包括当天需要最小交易额要求,后60天不做要求，否则认定为异常收益率

        all_date_list = list(this_coin_recent_amount.index)
        all_cal_date_list = [i for i in all_date_list if i not in kick_out_date]  # 部分日期不做交易额要求
        this_coin_recent_amount = this_coin_recent_amount[all_cal_date_list]
        small_amount_threshold = total_amount_series[all_cal_date_list] / 100000
        small_amount_threshold[small_amount_threshold < 20000] = 20000  # 计算出来最小交易额阈值小于20000的统统设置为20000

        if not ((this_coin_recent_amount - small_amount_threshold) >= 0).all():  # 如果当天交易额以及前后60天交易额有一个不满足最小交易额要求，这个收益率认为是异常收益率
            all_ret_df.loc[this_date, this_name] = np.nan  # 异常收益率直接填充为NaN
            temp_all_ret_df.loc[index, 'filter_or_not'] = True
        else:
            pass

    del this_coin_recent_amount
    del small_amount_threshold
    del split_df

    return all_ret_df, temp_all_ret_df


def all_true(series):
    if series.all():
        return 1
    else:
        return 0


def return_cleaning(coin_name, ico_date, ret, amount, amount_threshold):
    if type(ico_date) == float:
        filter_return = ret
        filter_return.name = coin_name
        return filter_return
    else:
        mask = pd.Series(index=ret.index)
        ico_data_6d = date_shifter(ico_date, step='days', how_many=6)
        ico_data_7d = date_shifter(ico_date, step='days', how_many=7)
        amount_diff = (amount - amount_threshold).loc[ico_data_7d:]
        amount_check = (amount_diff >= 0).rolling(60, min_periods=1).apply(all_true)
        mask[ico_date:ico_data_6d] = 1
        mask[amount_check.index] = amount_check
        mask.replace(0, np.nan, inplace=True)
        filter_return = ret * mask
        filter_return.name = coin_name
        return filter_return

if __name__ == '__main__':
    xlsx2csv_folder('E:\\tmp', 'E:\\data\\exchange_information\\margin_interest_rate')
    # filter_coins_not_updated_over_1year()
